from __future__ import division
import gzip
from HTMLParser import HTMLParser
from htmlentitydefs import name2codepoint
from urlparse import urlparse

class MyHTMLParser(HTMLParser):
    def handle_starttag(self, tag, attrs):
        # print "Start tag:", tag
        # set the current_tag to tag
        self.current_tag = tag
        
        if hasattr(self, 'doc_outlinks'):
            pass
        else:
            self.doc_outlinks = 0
        
        for attr in attrs:
            (attr_name, attr_value) = attr
            if attr_name == "href":
                #print "     attr:", attr
                self.doc_outlinks += 1
    def handle_endtag(self, tag):
        #print "End tag  :", tag
        # set back the current_tag to ""
        self.current_tag = ""
        
    def handle_data(self, data):
        if hasattr(self, 'current_tag'):
            if self.current_tag == "script":
                #print "The following content should NOT be indexed."
                #print "script content     :", data
                #print "script length     :",len(data)
                if hasattr(self, 'script_size'):
                    self.script_size += len(data)
                else:
                    self.script_size = len(data)
            else:
                if hasattr(self, 'text_size'):
                    self.text_size += len(data)
                else:
                    self.text_size = len(data)
                #print "text content     :", data

                for i in range(0,len(data)):
                    # print "data[i]:",ord(data[i])
                    if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
                        # Just replace them with a space.
                        data = data[:i] + " " + data[i+1:]
                
                # can index the data content now
                if hasattr(self, 'doc_words'):
                    pass
                else:
                    self.doc_words = []
                
                if hasattr(self, 'doc_distinct_words'):
                    pass
                else:
                    self.doc_distinct_words = []                
                    
                #print "raw word list:",data.split(" ")
                
                for word in data.split(" "):
                    if word != "":
                        self.doc_words.append(word)
                        if word not in self.doc_distinct_words:
                            self.doc_distinct_words.append(word)
                        
                #print "doc_words:",self.doc_words
                #print "doc_distinct_words:",self.doc_distinct_words
                  
                #print "text length      :", len(data)
        else:
            pass
    def handle_comment(self, data):
        pass
        #print "Comment  :", data
    def handle_entityref(self, name):
        pass
        #c = unichr(name2codepoint[name])
        #print "Named ent:", c
    def handle_charref(self, name):
        pass
        #if name.startswith('x'):
        #    c = unichr(int(name[1:], 16))
        #else:
        #    c = unichr(int(name))
        #print "Num ent  :", c
    def handle_decl(self, data):
        pass
        #print "Decl     :", data
    
    def generate_statistics_report(self,status):
        print "completed_parsed:",status
        # attrs check or init
        if hasattr(self, 'doc_words'):
            pass
        else:
            self.doc_words = []

        if hasattr(self, 'doc_distinct_words'):
            pass
        else:
            self.doc_distinct_words = []
        
        if hasattr(self, 'text_size'):
            pass
        else:
            self.text_size = 0
            
        if hasattr(self, 'script_size'):
            pass
        else:
            self.script_size = 0       

        if hasattr(self, 'doc_outlinks'):
            pass
        else:
            self.doc_outlinks = 0
            
        if hasattr(self, 'doc_waterloo_spam'):
            pass
        else:
            self.doc_waterloo_spam = 0
                
        print "doc_words:",len(self.doc_words)
        print "doc_distinct_words:",len(self.doc_distinct_words)
        print "text_size:",self.text_size
        print "script_size:",self.script_size
        script_text_ratio = self.script_size / self.text_size
        print "script_text_ratio:",script_text_ratio
        print "doc_outlinks:",self.doc_outlinks
        print "doc_waterloo_spam:","N/A"


print "The program begins..."
# load the 1M popular site lists.
inputAuxFileName = "/Users/weijiang/Documents/workspace/endlessFeatureGenerationTaskForMe/auxFiles/top-1m.head.2.csv"
inputAuxFileHandler = open(inputAuxFileName, "r")

top1MPopularSitesDict = {}

for line in inputAuxFileHandler.readlines():
    siteNumber = int( line.strip().split(",")[0] )
    siteDomain = line.strip().split(",")[1]
    if siteDomain not in top1MPopularSitesDict:
        top1MPopularSitesDict[siteDomain] = siteNumber
        
print "top1MPopularSitesDict:",top1MPopularSitesDict
    

inputFileName = "/Users/weijiang/Documents/workspace/endlessFeatureGenerationTaskForMe/auxFiles/cluewebOffcialEnglishIndexCategoryBPaths.experiment.simple.txt"
inputPathsFileHandler = open(inputFileName,"r")

docIDCounter = -1

for line in inputPathsFileHandler.readlines():
    # Here, should read the actual data compressed file, de-compress it and parse them one by one, generating features which someone want.
    # print "Processing:",line.strip()
    f = gzip.open(line.strip(), 'rb') 
    file_content = f.read()
    currentLine = ""
    #print "len(file_content):",len(file_content)
    
    numOfWARCFileHeader = 13
    numOfWebPageResponseHeader = 11
    numOfWebPageHTTPRequestHeader = 10
    
    contentLengthSpecialCounter = -1
    # print "compressed file content begin:"
    for i in range(0,len(file_content)):
    #for i in range(0,len(file_content)):
        if file_content[i] != "\n":
            currentLine += file_content[i]
        else:
            
            
            # WARC-TREC-ID: clueweb09-en0040-54-00000
            if currentLine.strip().split(":")[0].strip() == "WARC-TREC-ID":
                print "WARC-TREC-ID:",currentLine.strip().split(":")[1].strip()
            # WARC-Target-URI: http://www.smartwebby.com/DreamweaverTemplates/templates/business_general_template59.asp
            if currentLine.strip().split(":")[0].strip() == "WARC-Target-URI":
                target_URI = currentLine.strip()[17:]
                
                print "WARC-Target-URI:",target_URI
                # the logic of generating the url related features.
                url_length = len(target_URI)
                print "url_length:",url_length
                o = urlparse(target_URI)
                # print "str(o.path):",o.path
                # print "str(o.path).split('/'):",str(o.path).split('/')
                
                tempNumber = len(str(o.path).split('/'))
                if tempNumber == 0:
                    url_nesting = 0
                elif tempNumber == 1:
                    url_nesting = 0
                elif tempNumber == 2:
                    url_nesting = 0
                else:
                    url_nesting = tempNumber - 2
                
                url_host = str(o.netloc)[4:]
                
                url_in_top_million = False
                
                if url_host in top1MPopularSitesDict:
                    url_in_top_million = True
                else:
                    url_in_top_million = False
                
                print "url_in_top_million:",url_in_top_million
                
                print "url_nesting:",url_nesting
                
                print "url_domain:",str(o.netloc).split(".")[-1]
                
                print "url_in_top_million:","N/A"
                
                
                
                contentLengthSpecialCounter = 2
            if currentLine.strip().split(":")[0].strip() == "Content-Length":
                contentLengthSpecialCounter -= 1
                if contentLengthSpecialCounter == 0:
                    web_page_content_length = int(currentLine.strip()[16:])
                    print "doc_size:",currentLine.strip()[16:]
                    
                    docIDCounter += 1
                    print "doc_id:",docIDCounter
                    
                    while file_content[i] == "\n":
                        i += 1
                        
                    web_page_content = file_content[i:i+web_page_content_length]
                    
                    '''
                    outputFileName = str(docIDCounter) + ".txt"
                    outputFileHandler = open(outputFileName,"w")
                    outputFileHandler.write(web_page_content)
                    outputFileHandler.close()
                    '''
                    
                    parser = MyHTMLParser()
                    
                    try:
                        parser.feed(web_page_content)
                        parser.generate_statistics_report(1)
                    except:
                        # do not handle the error message.
                        parser.generate_statistics_report(0)
                    print
                    
                    # It is time to grap the pages.       
            currentLine = ""
    # print "compressed file content end."    
    f.close()
    
    
    
